/**********************************************************************/
/*
   Author: Michelle Han
   Created: 30 September, 2022
   Description: Merges and reshapes PMO data transfers:
		5. JPAL_DATA_8: Additional variables for batch 1 - 22
	 Outputs data at person level.
   Note that this code works with PII data (not deidentified).

   Note: to set filepaths, run MASTER.do.

   Output:
	 pmo_b1-22_raw_add_vars_wide.dta

*/
/**********************************************************************/

	* Set Filepaths
	if "$master_run" !="1" include "./Do/SET_FILEPATHS.do"
	* Log
	cap log close
	local prefix: display %tdCYND td(`c(current_date)')
	log using "$KP_logs/`prefix'_PMO_merge_wide_add_vars_b1-22.txt", replace text

/*----------------------------------------------------*/
            /* Section: Append batches */
/*----------------------------------------------------*/

* load first batch
	u "$KP_deid_admin/Raw/JPAL_DATA_8/additional variable_1.dta", clear
	gisid anon_id4
	gen batch = 1

* loop to append each batch
* Batch 18-22 does not have code_domisili, instead province_code/name, city_code/name
* Batch 18-22: final_test_score instead of test_score
* Batch 18:22 ticket_score instaed of bobot
	qui forval i = 2 / 22 {
		noi di "Appending Batch `i'"
		preserve
			u "$KP_deid_admin/Raw/JPAL_DATA_8/additional variable_`i'.dta", clear
			gen batch = `i'
			tempfile `i'
			qui save ``i''
		restore
		append using ``i''
	}

* Check uniqueness
	gisid anon_id4 batch
	gsort anon_id4 batch
	tab batch

* Check completeness
	mdesc

	/*---------------------------------------------------*/
	                  /* Section: Checks */
	/*----------------------------------------------------*/

* is this the first batch individual applies in?
	gsort anon_id4
	by anon_id4: gegen first_apply_batch = min(batch)

* check that if user gets accepted, no longer applies
	gsort anon_id4 batch
	by anon_id4 (batch): gen appearance_num = _n
	by anon_id4: gen appearances = _N
	by anon_id4 (batch): gen last_appearance = _n == _N
	gen select_batch_not_last = appearance_num != appearances if has_passed_current_batch == 1
	tab select_batch_not_last

	gsort anon_id4 batch
	by anon_id4: egen apply_after_selected = total(select_batch_not_last)
	tab apply_after_selected
	tab batch if apply_after_selected == 1

* check batch_sk is consistent across batches
	gsort anon_id4 batch_sk
	by anon_id4 (batch_sk): gen tag_batch_sk = batch_sk[1] != batch_sk[_N]


	/*----------------------------------------------------*/
	            /* Section: Reshape */
	/*----------------------------------------------------*/

* drop users that apply after being selected (111 individuals)
// NOTE: may change handling of this later
	drop if apply_after_selected == 1
	gsort anon_id4

* Prep to reshape wide
	gen applied = 1
	rename batch_sk batch_treated_39_
	compress

* Reshape wide
	local varlist applied ///
								has_passed_current_batch ///
								batch_treated_39_ ///
								date_batch ///
								date_incentive ///
								first_apply_batch ///
								status_revoked

	keep `varlist' anon_id4 batch

	greshape wide `varlist ', ///
		i(anon_id4) j(batch) nochecks benchmark

* Condense unnecessary variables
	foreach var in batch_treated_39_ first_apply_batch {
		egen `var' = rowfirst(`var'*)
		drop `var'?*
	}

* Status revoked is only equal to 1 in the batch that the applicant wins, so keep rowtotal
	gen status_revoked = .
	gen date_incentive = ""
	gen date_batch = ""

	qui forval i = 1 / 22 {
		replace status_revoked = status_revoked`i' if first_apply_batch == `i'
		replace date_incentive = date_incentive`i' if first_apply_batch == `i'
		replace date_batch = date_batch`i' if first_apply_batch == `i'
	}

	rename batch_treated_39_ batch_treated_39
	drop status_revoked?* date_batch?* date_incentive?*

* Save raw merged reshaped data
datasignature 
if "`r(datasignature)'" == "23391556:50(65783):3467617110:3160180872" {
   save "$KP_deid_admin/Clean/pmo_b1-22_raw_add_vars_wide.dta", replace
      }
else {
   di as err "Careful, your machine produces a different dataset"
   stop
		}
	

	cap log close


// DONE
